package pipeline;

import java.io.File;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.Hashtable;
import java.util.Random;
import java.util.Map.Entry;

import setting.BioNLPSettings;
import util.QuickSort;
import util.io.FileUtil;
import classifier.SVMWrapper;

public class CotrainTrigger extends TriggerDetectionSVM {
	
	public void learn(){
		str2id.clear();

		System.err.println("stuffing training data");
		BioNLPSettings.init("train.conf");
		ArrayList<Hashtable<String, Integer>> train = prepareSVMData(BioNLPSettings.getValue("dataPath"), BioNLPSettings.getValue("parsePath"), true);
		System.err.println("stuffing dev data");
		BioNLPSettings.init("dev.conf");
		ArrayList<Hashtable<String, Integer>> dev = prepareSVMData(BioNLPSettings.getValue("dataPath"), BioNLPSettings.getValue("parsePath"), true);
		System.err.println("stuffing test data");
		BioNLPSettings.init("test.conf");
		ArrayList<Hashtable<String, Integer>> test = prepareSVMData(BioNLPSettings.getValue("dataPath"), BioNLPSettings.getValue("parsePath"), true);
		
		
		System.err.println("encoding test...");
		for (Hashtable<String ,Integer> ex: test){
			for (String key: ex.keySet()){
				if (key.startsWith("SRC"))
					continue;
				if (!str2id.containsKey(key)){
					str2id.put(key, str2id.size()+1);
				}
			}
		}

		System.err.println("encoding dev...");
		for (Hashtable<String ,Integer> ex: dev){
			for (String key: ex.keySet()){
				if (key.startsWith("SRC"))
					continue;
				if (!str2id.containsKey(key)){
					str2id.put(key, str2id.size()+1);
				}
			}
		}
		
		System.err.println("encoding train...");
		for (Hashtable<String ,Integer> ex: train){
			for (String key: ex.keySet()){
				if (key.startsWith("SRC"))
					continue;
				if (!str2id.containsKey(key)){
					str2id.put(key, str2id.size()+1);
				}
			}
		}

		// feature dictionary
		BioNLPSettings.init("train.conf");
		StringBuffer sb = new StringBuffer();
//		if (!new File(BioNLPSettings.getValue("triggerFeatureMap")).exists())
		{
			for (Entry<String, Integer> entry: str2id.entrySet()){
				sb.append(entry.getKey()+"\t"+entry.getValue()+"\n");
			}
			FileUtil.writeTextToFile(sb.toString(), BioNLPSettings.getValue("triggerFeatureMap"));
		}
		/*else{
			String[] lines = FileUtil.getTextFromFile("triggerFeatureMap").split("\n");
			System.out.println("reading ... "+lines.length +" feature map");
			for (String line: lines){
				String[] pair = line.split("\t");
				str2id.put(pair[0], Integer.parseInt(pair[1]));
			}
		}*/

		printSVMFile(dev, "dev_trigger_view0.svm", "dev_trigger_view1.svm");
		printSVMFile(test, "test_trigger_view0.svm", "test_trigger_view1.svm");
		printSVMFile(train, "train_trigger_view0.svm", "train_trigger_view1.svm");
		
		// training svm models
		SVMWrapper svm = new SVMWrapper();
		String view0 = "_view0", view1 = "_view1";
		
		
		// use test data as unlabeled
		ArrayList<String> unlabeled0 = new ArrayList<String>(), unlabeled1 = new ArrayList<String>(), src = new ArrayList<String>();
		String[] instances = FileUtil.getTextFromFile( "dev_trigger_view0.svm").split("\n");
		for (String instance : instances){
			unlabeled0.add(instance);
		}
		instances = FileUtil.getTextFromFile( "dev_trigger_view1.svm").split("\n");
		for (String instance : instances){
			unlabeled1.add(instance);
		}
		instances = FileUtil.getTextFromFile( "dev_trigger_src.svm").split("\n");
		for (String instance : instances){
			src.add(instance);
		}
		ArrayList<Integer> idxPool = new ArrayList<Integer>();
		HashSet<Integer> idxVisited = new HashSet<Integer>();
		ArrayList<String> newlabel0 = new ArrayList<String>(), newlabel1 = new ArrayList<String>();
		Random rand = new Random(1230995);
		for (int k = 0; k < K; k++){
			System.err.println("[TRIGGER_COTRAIN] iteration "+k);
			
			// replenish pool
			while (idxPool.size() < U){
				int idx = rand.nextInt(unlabeled0.size());
				while (idxPool.contains(idx)||idxVisited.contains(idx)){
					idx = (idx+1) % unlabeled0.size();
				}
				idxPool.add(idx);
			}

			// retrain both views
			// add new labeled data
			HashSet<Integer> addon = new HashSet<Integer>();
			addon.addAll(cotrain(view0, newlabel0, idxPool, unlabeled0, svm, k));
			ArrayList<Integer> list = cotrain(view1, newlabel1, idxPool, unlabeled1, svm, k);
			for (Integer ii: list){
				if (ii > 0){
					if (addon.contains(-ii)){
						// don't agree
						addon.remove(-ii);
						System.err.println("[TRIGGER_COTRAIN] two views don't agree @"+(Math.abs(ii)-1)+"="+src.get(idxPool.get(Math.abs(ii)-1)));
					}
					else{
						addon.add(ii);
					}
				}
				else if (ii < 0){
					// resolve disagreement
					if (addon.contains(-ii)){
						addon.remove(-ii);
						System.err.println("[TRIGGER_COTRAIN] two views don't agree @"+(Math.abs(ii)-1)+"="+src.get(idxPool.get(Math.abs(ii)-1)));
					}
					else{
						addon.add(ii);
					}
				}
			}
			
			// update pool
			System.err.println("[TRIGGER_COTRAIN] adding "+addon.size()+" new examples");
			for (Integer idx0: addon){
				// idx  is the index of pool
				int idx = Math.abs(idx0)-1;
				
				idxVisited.add(idxPool.get(idx));
				String example = unlabeled0.get(idxPool.get(idx));
				int trueLabel = Integer.parseInt(example.substring(0,example.indexOf(" ")).replace("+",""));
				if (trueLabel * idx0 < 0){
					System.err.println("[TRIGGER_COTRAIN] wrong prediction with high confidence @"+(idx)+"="+src.get(idxPool.get(idx)));
				}
				if (idx0 >0){
					example = "1"+example.substring(example.indexOf(" "));
				}
				else{
					example = "-1"+example.substring(example.indexOf(" "));
				}
				newlabel0.add(example);
				example = unlabeled1.get(idxPool.get(idx));
				if (idx0 >0){
					example = "1"+example.substring(example.indexOf(" "));
				}
				else{
					example = "-1"+example.substring(example.indexOf(" "));
				}
				newlabel1.add(example);
			}
			for (Integer idx: idxVisited){
				// idx is the content of pool
				idxPool.remove(idx);
			}
		}
		
		
		
	}
	
	/**
	 * train over original data + newlabel;
	 * test over idxPool-specific data;
	 * pick up most confident P and N examples to addon (return result)
	 * test over dev set to give numbers;
	 * @param view
	 * @param newlabel0
	 * @param idxPool
	 * @param unlabeled0
	 * @param svm
	 * @return
	 */
	private ArrayList<Integer> cotrain(String view, ArrayList<String> newlabel, ArrayList<Integer> idxPool, ArrayList<String> unlabeled, SVMWrapper svm, int k) {
		System.err.println("[TRG_COTRAIN]training svm "+view);
		String[] sample = FileUtil.getTextFromFile("train_trigger"+view+".svm").split("\n");
		StringBuilder text = new StringBuilder();
		Random r = new Random(12309985);
		int num = 0;
		for (String s: sample){
			if (r.nextDouble() < 0.05){
				text.append(s+"\n");
				num ++;
			}
		}
		for (int i = 0; i < newlabel.size(); i++){
			text.append(newlabel.get(i)+"\n");
		}
		num+=newlabel.size();
		FileUtil.writeTextToFile(text.toString(), "train_trigger_pool"+view+".svm");
		
//		if (!new File(BioNLPSettings.getValue("TRIGGER_MODEL")+view).exists()){
		
		// retrain
			svm.train("train_trigger_pool"+view+".svm", BioNLPSettings.getValue("TRIGGER_MODEL")+view+".iter"+k);
//		}
//		else{
//			System.err.println("[TRG_Learn]training svm skipped "+view);
//		}
			double[] scores= svm.test("train_trigger"+view+".svm", BioNLPSettings.getValue("TRIGGER_MODEL")+view+".iter"+k, 
					"svm_predictions");
			double  f1 =scores[1]*scores[2]*2 / (scores[1]+scores[2]);
			System.err.println(k+"@train("+num+")"+view+": precision = "+scores[1]+", recall = "+scores[2]+", f1 = "+f1);
			// test over dev set
		System.err.println("[TRG_Learn]testing dev svm "+view);
		 scores= svm.test("dev_trigger"+view+".svm", BioNLPSettings.getValue("TRIGGER_MODEL")+view+".iter"+k, 
				"dev_trigger_predictions"+view);
		f1 = scores[1]*scores[2]*2 / (scores[1]+scores[2]);
		System.err.println(k+"@dev"+view+": precision = "+scores[1]+", recall = "+scores[2]+", f1 = "+f1);
		
		//pick up new unlabeled instances to label
		System.err.println("[TRG_Learn]testing test svm "+view);
		text = new StringBuilder();
		for (int i = 0; i < idxPool.size(); i++){
			text.append(unlabeled.get(idxPool.get(i))+"\n");
		}
		FileUtil.writeTextToFile(text.toString(), "test_pool_trigger"+view+".svm");
		svm.test("test_pool_trigger"+view+".svm",BioNLPSettings.getValue("TRIGGER_MODEL")+view+".iter"+k, 
				"test_pool_trigger_predictions"+view);
		
		ArrayList<Integer> addon = new ArrayList<Integer>();
		String[] lines = FileUtil.getTextFromFile("test_pool_trigger_predictions"+view).split("\n");
		scores = new double[lines.length];
		for (int i = 0; i < lines.length;i ++){
			scores[i] = Double.parseDouble(lines[i]);
		}
		int[] idx = QuickSort.quicksort(scores);
		System.err.println("[TRIGGER_COTRAIN] adding prediction labels");
		for (int i = 0; i < N; i++){
			
			if (scores[i] > -0.5)
				break;
			
			System.err.println("[TRIGGER_COTRAIN] adding negative with confidence "+idx[i]+" = "+scores[i]);
			addon.add(-idx[i]-1);
			
		}
		for (int i = 0; i < P; i++){
			if (scores[idx.length - 1 - i] < 0.5)
				break;
			System.err.println("[TRIGGER_COTRAIN] adding positive with confidence "+idx[idx.length - 1 - i]+" = "+scores[idx.length - 1 - i]);
			addon.add(idx[idx.length - 1 - i]+1);
		}
		return addon;
	}

	/**
	 * two views, 0 - non-dep, 1- dep
	 * @param data
	 * @param filename0
	 * @param filename1
	 */
	public void printSVMFile(ArrayList<Hashtable<String ,Integer>> data, String filename0, String filename1){
		System.err.println("printing " + filename0 +"...");
		StringBuilder sb0 = new StringBuilder(), sb1 = new StringBuilder(), src = new StringBuilder();
		for (Hashtable<String ,Integer> ex: data){
			ArrayList<Integer> features0 = new ArrayList<Integer>(), features1 = new ArrayList<Integer>();
			boolean tclass = true;
			for (String key: ex.keySet()){
				if (key.startsWith("SRC")){
					src.append(key+"\n");
					continue;
				}
				if (key.equals("NEG_CLASS"))
					tclass = false;
				else{
					if (str2id.containsKey(key)){
						if (key.contains("DEP")){
							int fid = str2id.get(key);
							if (!features1.contains(fid))
								features1.add(fid);
						}
						else{
							int fid = str2id.get(key);
							if (!features0.contains(fid))
								features0.add(fid);
						}
					}
				}
			}
			Collections.sort(features0);
			Collections.sort(features1);
			if (tclass){
				sb0.append("+1");
				sb1.append("+1");
			}
			else{
				sb0.append("-1");
				sb1.append("-1");
			}
			float weight = 1/(float)features0.size();
			for (int i = 0; i < features0.size(); i++){
				sb0.append(" "+features0.get(i)+":1");
			}
			sb0.append("\n");
			weight = 1/(float)features1.size();
			for (int i = 0; i < features1.size(); i++){
				sb1.append(" "+features1.get(i)+":1");
			}
			sb1.append("\n");
		}
		FileUtil.writeTextToFile(src.toString(), filename0.replace("_view0","_src"));
		FileUtil.writeTextToFile(sb0.toString(), filename0);
		FileUtil.writeTextToFile(sb1.toString(), filename1);
	}
	
	// K=#iter, P=#pos pick, N= #neg pick, U= # pool
	
	public static int K = 150, P = 20, N = 200, U = 3000; 
}
